import urllib2
import sys
import re
import Queue
import time
import threading
import traceback

from BeautifulSoup import BeautifulSoup

from structures import Product, Bug, Change
from functions import Logger, login, string_to_date, download_products, opener
from functions import all_products, all_bugs, all_changes, bug_ids, missing_bugs
from functions import URL, TIMEOUT, engine, Base, session

# urllib2 initialization
urllib2.install_opener(opener)

# Global variables
queue = Queue.Queue()
out_queue = Queue.Queue()
errors = set()

date = re.compile(r'\d\d\d\d-\d\d-\d\d')

class BugParser(threading.Thread):
    def __init__(self, queue, out_queue):
        threading.Thread.__init__(self)
        self.queue = queue
        self.out_queue = out_queue
    def run(self):
        while True:
            bugid, savedbugs, savedchanges = self.queue.get()
            print('Downloading bug {}...'.format(bugid))
            if bugid not in savedbugs:
                try:
                    f = opener.open(URL + 'show_bug.cgi?ctype=xml&id={}'.format(
                        bugid), timeout=TIMEOUT)
                    data = f.read()
                    f.close()
                    soup = BeautifulSoup(data)
                    product = soup.product.string.replace('&amp;', '&')
                    opened = string_to_date(soup.creation_ts.string)
                    resolution = soup.resolution.string
                    status = soup.bug_status.string
                    category = soup.cf_category.string
                    modified = string_to_date(soup.delta_ts.string)
                    priority = int(soup.priority.string[1])
                    severity = soup.bug_severity.string
                    bug = Bug(bugid, product, opened, resolution, status,
                        category, modified, priority, severity)
                    bug, changes = self.update_bug(bug)
                    self.out_queue.put((bug, changes))
                except:
                    print('...bug {} not downloaded. Error: {}'.format(bugid,
                        sys.exc_info()[0]))
                    traceback.print_exc()
                    errors.add(bugid)
                finally:
                    self.queue.task_done()
            else:
                bug, changes = self.update_bug(savedbugs[bugid], savedchanges)
                self.out_queue.put((bug, changes))
                self.queue.task_done()
    def update_bug(self, bug, savedchanges=None):
        bugid = int(bug.bugid)
        if savedchanges:
            changelist = [change for change in savedchanges
                if change.bugid == bugid]
        else:
            changelist = []
        newchanges = []
        f = opener.open(URL + 'show_bug.cgi?ctype=xml&id={}'.format(bugid),
            timeout=TIMEOUT)
        data = f.read()
        f.close()
        soup = BeautifulSoup(data)
        product = soup.product.string.replace('&amp;', '&')
        resolution = soup.resolution.string
        status = soup.bug_status.string
        category = soup.cf_category.string
        modified = string_to_date(soup.delta_ts.string)
        priority = int(soup.priority.string[1])
        severity = soup.bug_severity.string

        f = opener.open(URL + 'show_activity.cgi?id={}'.format(bugid),
            timeout=TIMEOUT)
        data = f.read()
        f.close()
        soup = BeautifulSoup(data)
        table = soup('table')[2]
        rows = table('tr')
        who = ''
        when = ''
        what = ''
        removed = ''
        added = ''
        try:
            for i in rows[1:]:
                column = i('td')
                if len(column) == 5:
                    who = column[0].find(text=True).strip().replace('&#64;',
                        '@')
                    when = string_to_date(re.findall(date, column[1].find(
                        text=True).strip())[0])
                    what = column[2].find(text=True).strip()
                    removed = column[3].find(text=True).strip().replace(
                        '&#64;', '@').replace('&amp;', '&')
                    added = column[4].find(text=True).strip().replace(
                        '&#64;', '@').replace('&amp;', '&')
                elif len(column) == 3:
                    what = column[0].find(text=True).strip()
                    removed = column[1].find(text=True).strip().replace(
                        '&#64;', '@').replace('&amp;', '&')
                    added = column[2].find(text=True).strip().replace(
                        '&#64;', '@').replace('&amp;', '&')
                change = Change(who, when, what, removed, added)
                if not change in changelist:
                    newchanges.append(change)
        except IndexError:
            traceback.print_exc()
            errors.add(bugid)
        bug.life()
        return (bug, newchanges)

class ProcessBug(threading.Thread):
    def __init__(self, queue):
        threading.Thread.__init__(self)
        self.queue = queue
    def run(self):
        while True:
            bug, changes = self.queue.get()
            print('Saving bug {}...'.format(bug.bugid))
            try:
                if bug:
                    session.add(bug)
                    bug.changes.extend(changes)
            except:
                print('...bug {} not saved. Error: {}'.format(bug.bugid,
                    sys.exc_info()[0]))
                traceback.print_exc()
                errors.add(bug.bugid)
            finally:
                self.queue.task_done()

def download(bugids):
    errors = set()
    start = time.time()
    try:
        savedbugs = all_bugs()
        savedbugids = [bug.bugid for bug in savedbugs]
        savedbugs = dict(zip(savedbugids, savedbugs))
        savedchanges = all_changes()
    except:
        savedbugs = []
        savedchanges = []
    for bugid in bugids:
        queue.put((bugid, savedbugs, savedchanges))
    queue.join()
    print('Downloading complete!')
    out_queue.join()
    print('Saving complete!')
    session.commit()
    t = (time.time() - start)
    print('Elapsed Time: {} seconds ({} seconds per bug)'.format(t,
        t*1.0/len(bugids)))
    if errors:
        print('The following bugs had errors and were not completed: {}'.format(
            str(errors)))

if __name__ == '__main__':
    sys.stdout = Logger()
    Base.metadata.create_all(engine)
    if login():
        for i in range(20):
            t = BugParser(queue, out_queue)
            t.setDaemon(True)
            t.start()
        for i in range(5):
            pt = ProcessBug(out_queue)
            pt.setDaemon(True)
            pt.start()
        #download(range(1, 2))


